import java.sql.*;

import javax.sql.*;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.InputStreamReader;

import java.io.IOException;

import java.util.regex.*;

import java.lang.*;
import java.util.*;
import java.util.Date;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.StringEscapeUtils;

public class GetData
{
	//the file to store email suffix
	String dstFile = null;
	
	//db object
	DB db = null;
	
	//store distributed map
	private HashMap<String, Integer> emailSurrixDeistributionMap = null;
	
	public void setDestFile(String dstFile)
	{
		this.dstFile = dstFile;
	}
	
	public GetData(String dstFile) throws ClassNotFoundException, 
		IllegalAccessException, SQLException
	{
		//get the filepath
		this.dstFile = dstFile;
		
		//connect to the db
		db = new DB();
		db.connect();
	}
	
	private void getEmailSuffix(int rowNumLimit) 
			throws SQLException, IOException
	{
		//instantiate the distribution hash map
		emailSurrixDeistributionMap = new HashMap<String, Integer>();
		
		//sql statement that counts the number of rows in comment given bugid
		String countDescriptionStatement = 
				"select author from comment where author is not null";
		
		//prepared statement object for comment
		PreparedStatement countEmail = null;
		
		try
		{
			//get the prepared statement
			countEmail = db.getStreamPreparedStatement(countDescriptionStatement, 
					rowNumLimit);			
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		
		//retrive email suffix
		ResultSet descriptionRs = countEmail.executeQuery();
		while (descriptionRs.next())
		{
			String email = descriptionRs.getString(1);
			if (email != null)
			{
				//unescap the html coding
				email = StringEscapeUtils.unescapeHtml(email);
				
				String emailSuffix = email.substring(email.lastIndexOf('@') + 1,
						email.length());
				if (email.lastIndexOf('@') > -1)
				{
					addMap(emailSuffix, emailSurrixDeistributionMap);		
				}
			}
			
		}
		
		int totalNum = 0;
		
		System.out.println("here comes suffix");
		
		Set<String> emailSuffixList = emailSurrixDeistributionMap.keySet();
		for (String emailSuffix: emailSuffixList)
		{
			int num = emailSurrixDeistributionMap.get(emailSuffix);
			System.out.println(emailSuffix + " --> " + num);
			totalNum += num;
			
			if (num > 4000)
			{
				System.err.println(emailSuffix);
				totalNum -= num;
			}
		}
		
		System.out.println("total: " + totalNum);
	}
	
	private void getDescription(int rowNumLimit) 
			throws SQLException, IOException
	{
		
		//sql statement that counts the number of rows in comment given bugid
		String countDescriptionStatement = 
				"select * from description where description is not null";
		
		//prepared statement object for comment
		PreparedStatement countDescription = null;
		
		try
		{
			//get the prepared statement
			countDescription = 
					db.getStreamPreparedStatement(countDescriptionStatement, 
							rowNumLimit);			
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		
		FileWriter fstream = new FileWriter(dstFile);
		
		BufferedWriter out = new BufferedWriter(fstream);
		
		
		//retrive email suffix
		ResultSet descriptionRs = countDescription.executeQuery();
		while (descriptionRs.next())
		{
			String description = descriptionRs.getString(2);
			int bugid = descriptionRs.getInt(1);
			if (description != null)
			{
				//unescap the html coding
				description = StringEscapeUtils.unescapeHtml(description);
				//description = StringEscapeUtils.unescapeHtml(description);
			
				description = description.replace(',', '.');
				description = StringUtils.remove(description, "\n");
				description = StringUtils.remove(description, "\"");
				description = StringUtils.remove(description, "\'");
				//System.out.println(StringUtils.remove(description, "\n"));
				out.write(bugid + ",\"" + description + "\"" + "\n");
			}
			
		}
		
		out.close();
		
	}
	
	//add current size into the distribution map
	private void addMap(String key, HashMap<String, Integer> distributionMap)
	{
		//if this size exists, just increase it by 1
		if (distributionMap.containsKey(key))
		{
			int num = distributionMap.get(key);
			
			//need to remove it first, inefficient :(
			distributionMap.remove(key);
			
			distributionMap.put(key, num + 1);
		}
		//else add it into this map, set the num as 1
		else
		{
			distributionMap.put(key, 1);
		}
	}
	
	private void writeTopicDistribution(String srcFile, String dstFile, 
			double threshold, int topicNum) throws IOException
	{
		FileWriter ofstream = new FileWriter(dstFile);	
		BufferedWriter out = new BufferedWriter(ofstream);
		
		FileInputStream ifstream = new FileInputStream(srcFile);
		DataInputStream in = new DataInputStream(ifstream);
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		
		String line = new String();
		
		while ((line = br.readLine()) != null)   
		{
			String[] words = line.split("\\W");
			
			out.write(words[0]);
			
			int topicCount = 0;
			
			for (int i = 1; i <= topicNum; i++)
			{
				//System.out.println(Double.parseDouble(words[i]));
				
				if (Double.parseDouble(words[i]) >= threshold)
				{
					topicCount++;
				}
			}
			
			out.write(", " + topicCount + "\n");
		}
		
		out.close();
		in.close();
	}
	
	public static void main(String args[]) throws ClassNotFoundException, 
	IllegalAccessException, SQLException, IOException
	{
		GetData description = new GetData("data/emailStatistics.csv");
		
		//description.getDescription(500);
		description.writeTopicDistribution("data/document-topic-distributions.csv", 
				"data/topicProbability.csv", 0.0001, 30);
	}
}
